#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2018-8-15 12:00 
# @Author : By Joker
# @File : getBook2.py 
# @Software: PyCharm

from urllib import request

from bs4 import BeautifulSoup


# 获取指定url中的正文
def download_specified_chapter(chapter_url, header, coding):
    # 生成request对象
    dowload_req = request.Request(chapter_url, headers=header)
    # 打开request对象中的url网址,并获得对应内容
    response = request.urlopen(dowload_req)
    # 解码之后,获取页面的html
    download_html = response.read().decode(coding, 'ignore')
    # 获取html的beautifulSoup
    origin_soup = BeautifulSoup(download_html, 'lxml')
    # 获取正文部分
    content = origin_soup.find(class_='content')
    # 整理正文格式
    txt = content.text.replace('\n' * 8, '\n').replace('nrgg360();', '')
    print(txt)


# 获取指定url中的目录列表
def download_all_url(mail_url, header, coding):
    url_req = request.Request(mail_url, headers=header)
    response = request.urlopen(url_req)
    html = response.read().decode(coding, "ignore")
    html_soup = BeautifulSoup(html, "lxml")
    # url_con = html_soup.find_all("li",class_="line2")
    # print(url_con)
    # 整理正文
    for element in html_soup.find_all("li", class_="line2"):
        chapter_name = element.string
        chapter_url = "http://www.75xs.cc" + element.a.get('href')
        print("{} 链接: {}".format(chapter_name, chapter_url))


if __name__ == "__main__":
    # target_url = 'http://www.75xs.cc/book/longzu1/1.html'
    # header = {
    #     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36 Maxthon/5.2.3.4000'}
    # download_specified_chapter(target_url, header, 'utf-8')

    target_url = 'http://www.75xs.cc/book/longzu1/'
    header = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36 Maxthon/5.2.3.4000'}
    download_all_url(target_url, header, 'utf-8')
